<!DOCTYPE html>

<html lang="en">

<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <meta name="mobile-web-app-capable" content="yes">
    <title>
        Implementation of AlexNet - HackMD
    </title>
    <link rel="icon" type="image/png" href="https://hackmd.io/favicon.png">
    <link rel="apple-touch-icon" href="https://hackmd.io/apple-touch-icon.png">

    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/ionicons/2.0.1/css/ionicons.min.css" integrity="sha256-3iu9jgsy9TpTwXKb7bNQzqWekRX7pPK+2OLj3R922fo=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/octicons/3.5.0/octicons.min.css" integrity="sha256-QiWfLIsCT02Sdwkogf6YMiQlj4NE84MKkzEMkZnMGdg=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.5.1/themes/prism.min.css" integrity="sha256-vtR0hSWRc3Tb26iuN2oZHt3KRUomwTufNIf5/4oeCyg=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@hackmd/emojify.js@2.1.0/dist/css/basic/emojify.min.css" integrity="sha256-UOrvMOsSDSrW6szVLe8ZDZezBxh5IoIfgTwdNDgTjiU=" crossorigin="anonymous" />
    <style>
        @charset "UTF-8";@import url(https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,500,500i|Source+Code+Pro:300,400,500|Source+Sans+Pro:300,300i,400,400i,600,600i|Source+Serif+Pro&subset=latin-ext);.hljs{display:block;background:#fff;padding:.5em;color:#333;overflow-x:auto}.hljs-comment,.hljs-meta{color:#969896}.hljs-emphasis,.hljs-quote,.hljs-string,.hljs-strong,.hljs-template-variable,.hljs-variable{color:#df5000}.hljs-keyword,.hljs-selector-tag,.hljs-type{color:#a71d5d}.hljs-attribute,.hljs-bullet,.hljs-literal,.hljs-number,.hljs-symbol{color:#0086b3}.hljs-built_in,.hljs-builtin-name{color:#005cc5}.hljs-name,.hljs-section{color:#63a35c}.hljs-tag{color:#333}.hljs-attr,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-selector-pseudo,.hljs-title{color:#795da3}.hljs-addition{color:#55a532;background-color:#eaffea}.hljs-deletion{color:#bd2c00;background-color:#ffecec}.hljs-link{text-decoration:underline}.markdown-body{font-size:16px;line-height:1.5;word-wrap:break-word}.markdown-body:after,.markdown-body:before{display:table;content:""}.markdown-body:after{clear:both}.markdown-body>:first-child{margin-top:0!important}.markdown-body>:last-child{margin-bottom:0!important}.markdown-body a:not([href]){color:inherit;text-decoration:none}.markdown-body .absent{color:#c00}.markdown-body .anchor{float:left;padding-right:4px;margin-left:-20px;line-height:1}.markdown-body .anchor:focus{outline:none}.markdown-body blockquote,.markdown-body dl,.markdown-body ol,.markdown-body p,.markdown-body pre,.markdown-body table,.markdown-body ul{margin-top:0;margin-bottom:16px}.markdown-body hr{height:.25em;padding:0;margin:24px 0;background-color:#e7e7e7;border:0}.markdown-body blockquote{font-size:16px;padding:0 1em;color:#777;border-left:.25em solid #ddd}.markdown-body blockquote>:first-child{margin-top:0}.markdown-body blockquote>:last-child{margin-bottom:0}.markdown-body kbd,.popover kbd{display:inline-block;padding:3px 5px;font-size:11px;line-height:10px;color:#555;vertical-align:middle;background-color:#fcfcfc;border:1px solid #ccc;border-bottom-color:#bbb;border-radius:3px;box-shadow:inset 0 -1px 0 #bbb}.markdown-body .loweralpha{list-style-type:lower-alpha}.markdown-body h1,.markdown-body h2,.markdown-body h3,.markdown-body h4,.markdown-body h5,.markdown-body h6{margin-top:24px;margin-bottom:16px;font-weight:600;line-height:1.25}.markdown-body h1 .octicon-link,.markdown-body h2 .octicon-link,.markdown-body h3 .octicon-link,.markdown-body h4 .octicon-link,.markdown-body h5 .octicon-link,.markdown-body h6 .octicon-link{color:#000;vertical-align:middle;visibility:hidden}.markdown-body h1:hover .anchor,.markdown-body h2:hover .anchor,.markdown-body h3:hover .anchor,.markdown-body h4:hover .anchor,.markdown-body h5:hover .anchor,.markdown-body h6:hover .anchor{text-decoration:none}.markdown-body h1:hover .anchor .octicon-link,.markdown-body h2:hover .anchor .octicon-link,.markdown-body h3:hover .anchor .octicon-link,.markdown-body h4:hover .anchor .octicon-link,.markdown-body h5:hover .anchor .octicon-link,.markdown-body h6:hover .anchor .octicon-link{visibility:visible}.markdown-body h1 code,.markdown-body h1 tt,.markdown-body h2 code,.markdown-body h2 tt,.markdown-body h3 code,.markdown-body h3 tt,.markdown-body h4 code,.markdown-body h4 tt,.markdown-body h5 code,.markdown-body h5 tt,.markdown-body h6 code,.markdown-body h6 tt{font-size:inherit}.markdown-body h1{font-size:2em}.markdown-body h1,.markdown-body h2{padding-bottom:.3em;border-bottom:1px solid #eee}.markdown-body h2{font-size:1.5em}.markdown-body h3{font-size:1.25em}.markdown-body h4{font-size:1em}.markdown-body h5{font-size:.875em}.markdown-body h6{font-size:.85em;color:#777}.markdown-body ol,.markdown-body ul{padding-left:2em}.markdown-body ol.no-list,.markdown-body ul.no-list{padding:0;list-style-type:none}.markdown-body ol ol,.markdown-body ol ul,.markdown-body ul ol,.markdown-body ul ul{margin-top:0;margin-bottom:0}.markdown-body li>p{margin-top:16px}.markdown-body li+li{padding-top:.25em}.markdown-body dl{padding:0}.markdown-body dl dt{padding:0;margin-top:16px;font-size:1em;font-style:italic;font-weight:700}.markdown-body dl dd{padding:0 16px;margin-bottom:16px}.markdown-body table{display:block;width:100%;overflow:auto;word-break:normal;word-break:keep-all}.markdown-body table th{font-weight:700}.markdown-body table td,.markdown-body table th{padding:6px 13px;border:1px solid #ddd}.markdown-body table tr{background-color:#fff;border-top:1px solid #ccc}.markdown-body table tr:nth-child(2n){background-color:#f8f8f8}.markdown-body img{max-width:100%;box-sizing:content-box;background-color:#fff}.markdown-body img[align=right]{padding-left:20px}.markdown-body img[align=left]{padding-right:20px}.markdown-body .emoji{max-width:none;vertical-align:text-top;background-color:transparent}.markdown-body span.frame{display:block;overflow:hidden}.markdown-body span.frame>span{display:block;float:left;width:auto;padding:7px;margin:13px 0 0;overflow:hidden;border:1px solid #ddd}.markdown-body span.frame span img{display:block;float:left}.markdown-body span.frame span span{display:block;padding:5px 0 0;clear:both;color:#333}.markdown-body span.align-center{display:block;overflow:hidden;clear:both}.markdown-body span.align-center>span{display:block;margin:13px auto 0;overflow:hidden;text-align:center}.markdown-body span.align-center span img{margin:0 auto;text-align:center}.markdown-body span.align-right{display:block;overflow:hidden;clear:both}.markdown-body span.align-right>span{display:block;margin:13px 0 0;overflow:hidden;text-align:right}.markdown-body span.align-right span img{margin:0;text-align:right}.markdown-body span.float-left{display:block;float:left;margin-right:13px;overflow:hidden}.markdown-body span.float-left span{margin:13px 0 0}.markdown-body span.float-right{display:block;float:right;margin-left:13px;overflow:hidden}.markdown-body span.float-right>span{display:block;margin:13px auto 0;overflow:hidden;text-align:right}.markdown-body code,.markdown-body tt{padding:0;padding-top:.2em;padding-bottom:.2em;margin:0;font-size:85%;background-color:rgba(0,0,0,.04);border-radius:3px}.markdown-body code:after,.markdown-body code:before,.markdown-body tt:after,.markdown-body tt:before{letter-spacing:-.2em;content:"\00a0"}.markdown-body code br,.markdown-body tt br{display:none}.markdown-body del code{text-decoration:inherit}.markdown-body pre{word-wrap:normal}.markdown-body pre>code{padding:0;margin:0;font-size:100%;word-break:normal;white-space:pre;background:transparent;border:0}.markdown-body .highlight{margin-bottom:16px}.markdown-body .highlight pre{margin-bottom:0;word-break:normal}.markdown-body .highlight pre,.markdown-body pre{padding:16px;overflow:auto;font-size:85%;line-height:1.45;background-color:#f7f7f7;border-radius:3px}.markdown-body pre code,.markdown-body pre tt{display:inline;max-width:auto;padding:0;margin:0;overflow:visible;line-height:inherit;word-wrap:normal;background-color:transparent;border:0}.markdown-body pre code:after,.markdown-body pre code:before,.markdown-body pre tt:after,.markdown-body pre tt:before{content:normal}.markdown-body .csv-data td,.markdown-body .csv-data th{padding:5px;overflow:hidden;font-size:12px;line-height:1;text-align:left;white-space:nowrap}.markdown-body .csv-data .blob-line-num{padding:10px 8px 9px;text-align:right;background:#fff;border:0}.markdown-body .csv-data tr{border-top:0}.markdown-body .csv-data th{font-weight:700;background:#f8f8f8;border-top:0}.news .alert .markdown-body blockquote{padding:0 0 0 40px;border:0 none}.activity-tab .news .alert .commits,.activity-tab .news .markdown-body blockquote{padding-left:0}.task-list-item{list-style-type:none}.task-list-item label{font-weight:400}.task-list-item.enabled label{cursor:pointer}.task-list-item+.task-list-item{margin-top:3px}.task-list-item-checkbox{float:left;margin:.31em 0 .2em -1.3em!important;vertical-align:middle;cursor:default!important}.markdown-body{padding-top:40px;padding-bottom:40px;max-width:758px;overflow:visible!important}.markdown-body .emoji{vertical-align:top}.markdown-body pre{border:inherit!important}.markdown-body code{color:inherit!important}.markdown-body pre code .wrapper{display:-moz-inline-flex;display:-ms-inline-flex;display:-o-inline-flex;display:inline-flex}.markdown-body pre code .gutter{float:left;overflow:hidden;-webkit-user-select:none;user-select:none}.markdown-body pre code .gutter.linenumber{text-align:right;position:relative;display:inline-block;cursor:default;z-index:4;padding:0 8px 0 0;min-width:20px;box-sizing:content-box;color:#afafaf!important;border-right:3px solid #6ce26c!important}.markdown-body pre code .gutter.linenumber>span:before{content:attr(data-linenumber)}.markdown-body pre code .code{float:left;margin:0 0 0 16px}.markdown-body .gist .line-numbers{border-left:none;border-top:none;border-bottom:none}.markdown-body .gist .line-data{border:none}.markdown-body .gist table{border-spacing:0;border-collapse:inherit!important}.markdown-body code[data-gist-id]{background:none;padding:0}.markdown-body code[data-gist-id]:after,.markdown-body code[data-gist-id]:before{content:""}.markdown-body code[data-gist-id] .blob-num{border:unset}.markdown-body code[data-gist-id] table{overflow:unset;margin-bottom:unset}.markdown-body code[data-gist-id] table tr{background:unset}.markdown-body[dir=rtl] pre{direction:ltr}.markdown-body[dir=rtl] code{direction:ltr;unicode-bidi:embed}.markdown-body .alert>p{margin-bottom:0}.markdown-body pre.abc,.markdown-body pre.flow-chart,.markdown-body pre.graphviz,.markdown-body pre.mermaid,.markdown-body pre.sequence-diagram,.markdown-body pre.vega{text-align:center;background-color:inherit;border-radius:0;white-space:inherit}.markdown-body pre.abc>code,.markdown-body pre.flow-chart>code,.markdown-body pre.graphviz>code,.markdown-body pre.mermaid>code,.markdown-body pre.sequence-diagram>code,.markdown-body pre.vega>code{text-align:left}.markdown-body pre.abc>svg,.markdown-body pre.flow-chart>svg,.markdown-body pre.graphviz>svg,.markdown-body pre.mermaid>svg,.markdown-body pre.sequence-diagram>svg,.markdown-body pre.vega>svg{max-width:100%;height:100%}.markdown-body pre>code.wrap{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.markdown-body .alert>p,.markdown-body .alert>ul{margin-bottom:0}.markdown-body summary{display:list-item}.markdown-body summary:focus{outline:none}.markdown-body details summary{cursor:pointer}.markdown-body details:not([open])>:not(summary){display:none}.markdown-body figure{margin:1em 40px}.markdown-body .mark,.markdown-body mark{background-color:#fff1a7}.vimeo,.youtube{cursor:pointer;display:table;text-align:center;background-position:50%;background-repeat:no-repeat;background-size:contain;background-color:#000;overflow:hidden}.vimeo,.youtube{position:relative;width:100%}.youtube{padding-bottom:56.25%}.vimeo img{width:100%;object-fit:contain;z-index:0}.youtube img{object-fit:cover;z-index:0}.vimeo iframe,.youtube iframe,.youtube img{width:100%;height:100%;position:absolute;top:0;left:0}.vimeo iframe,.youtube iframe{vertical-align:middle;z-index:1}.vimeo .icon,.youtube .icon{position:absolute;height:auto;width:auto;top:50%;left:50%;transform:translate(-50%,-50%);color:#fff;opacity:.3;transition:opacity .2s;z-index:0}.vimeo:hover .icon,.youtube:hover .icon{opacity:.6;transition:opacity .2s}.slideshare .inner,.speakerdeck .inner{position:relative;width:100%}.slideshare .inner iframe,.speakerdeck .inner iframe{position:absolute;top:0;bottom:0;left:0;right:0;width:100%;height:100%}.MJX_Assistive_MathML{display:none}.ui-infobar{position:relative;z-index:2;max-width:760px;margin:25px auto -25px;padding:0 15px;color:#777}.toc .invisable-node{list-style-type:none}.ui-toc{position:fixed;bottom:20px;z-index:998}.ui-toc-label{opacity:.3;background-color:#ccc;border:none;transition:opacity .2s}.ui-toc .open .ui-toc-label{opacity:1;color:#fff;transition:opacity .2s}.ui-toc-label:focus{opacity:.3;background-color:#ccc;color:#000}.ui-toc-label:hover{opacity:1;background-color:#ccc;transition:opacity .2s}.ui-toc-dropdown{margin-top:23px;margin-bottom:20px;padding-left:10px;padding-right:10px;max-width:45vw;width:25vw;max-height:70vh;overflow:auto;text-align:inherit}.ui-toc-dropdown>.toc{max-height:calc(70vh - 100px);overflow:auto}.ui-toc-dropdown[dir=rtl] .nav{padding-right:0;letter-spacing:.0029em}.ui-toc-dropdown a{overflow:hidden;text-overflow:ellipsis;white-space:pre}.ui-toc-dropdown .nav>li>a{display:block;padding:4px 20px;font-size:13px;font-weight:500;color:#767676}.ui-toc-dropdown .nav>li:first-child:last-child > ul,.ui-toc-dropdown .toc.expand ul{display:block}.ui-toc-dropdown .nav>li>a:focus,.ui-toc-dropdown .nav>li>a:hover{padding-left:19px;color:#000;text-decoration:none;background-color:transparent;border-left:1px solid #000}.ui-toc-dropdown[dir=rtl] .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav>li>a:hover{padding-right:19px;border-left:none;border-right:1px solid #000}.ui-toc-dropdown .nav>.active:focus>a,.ui-toc-dropdown .nav>.active:hover>a,.ui-toc-dropdown .nav>.active>a{padding-left:18px;font-weight:700;color:#000;background-color:transparent;border-left:2px solid #000}.ui-toc-dropdown[dir=rtl] .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav>.active>a{padding-right:18px;border-left:none;border-right:2px solid #000}.ui-toc-dropdown .nav .nav{display:none;padding-bottom:10px}.ui-toc-dropdown .nav>.active>ul{display:block}.ui-toc-dropdown .nav .nav>li>a{padding-top:1px;padding-bottom:1px;padding-left:30px;font-size:12px;font-weight:400}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a{padding-right:30px}.ui-toc-dropdown .nav .nav>li>ul>li>a{padding-top:1px;padding-bottom:1px;padding-left:40px;font-size:12px;font-weight:400}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a{padding-right:40px}.ui-toc-dropdown .nav .nav>li>a:focus,.ui-toc-dropdown .nav .nav>li>a:hover{padding-left:29px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:hover{padding-right:29px}.ui-toc-dropdown .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown .nav .nav>li>ul>li>a:hover{padding-left:39px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:hover{padding-right:39px}.ui-toc-dropdown .nav .nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>a{padding-left:28px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>a{padding-right:28px}.ui-toc-dropdown .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active>a{padding-left:38px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active>a{padding-right:38px}.markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,sans-serif}html[lang^=ja] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html[lang=zh-tw] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html[lang=zh-cn] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}html .markdown-body[lang^=ja]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html .markdown-body[lang=zh-tw]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html .markdown-body[lang=zh-cn]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}html[lang^=ja] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Meiryo UI,MS PGothic,ＭＳ\ Ｐゴシック,sans-serif}html[lang=zh-tw] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Microsoft JhengHei UI,微軟正黑UI,sans-serif}html[lang=zh-cn] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Microsoft YaHei UI,微软雅黑UI,sans-serif}html .ui-toc-dropdown[lang^=ja]{font-family:Source Sans Pro,Helvetica,Arial,Meiryo UI,MS PGothic,ＭＳ\ Ｐゴシック,sans-serif}html .ui-toc-dropdown[lang=zh-tw]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft JhengHei UI,微軟正黑UI,sans-serif}html .ui-toc-dropdown[lang=zh-cn]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft YaHei UI,微软雅黑UI,sans-serif}.ui-affix-toc{position:fixed;top:0;max-width:15vw;max-height:70vh;overflow:auto}.back-to-top,.expand-toggle,.go-to-bottom{display:block;padding:4px 10px;margin-top:10px;margin-left:10px;font-size:12px;font-weight:500;color:#999}.back-to-top:focus,.back-to-top:hover,.expand-toggle:focus,.expand-toggle:hover,.go-to-bottom:focus,.go-to-bottom:hover{color:#563d7c;text-decoration:none}.back-to-top,.go-to-bottom{margin-top:0}.ui-user-icon{width:20px;height:20px;display:block;border-radius:3px;margin-top:2px;margin-bottom:2px;margin-right:5px;background-position:50%;background-repeat:no-repeat;background-size:cover}.ui-user-icon.small{width:18px;height:18px;display:inline-block;vertical-align:middle;margin:0 0 .2em}.ui-infobar>small>span{line-height:22px}.ui-infobar>small .dropdown{display:inline-block}.ui-infobar>small .dropdown a:focus,.ui-infobar>small .dropdown a:hover{text-decoration:none}.ui-published-note{color:#337ab7}.ui-published-note .fa{font-size:20px;vertical-align:top}.unselectable{-webkit-user-select:none;-o-user-select:none;user-select:none}@media print{blockquote,div,img,pre,table{page-break-inside:avoid!important}a[href]:after{font-size:12px!important}}.markdown-body.slides{position:relative;z-index:1;color:#222}.markdown-body.slides:before{content:"";display:block;position:absolute;top:0;left:0;right:0;bottom:0;z-index:-1;background-color:currentColor;box-shadow:0 0 0 50vw}.markdown-body.slides section[data-markdown]{position:relative;margin-bottom:1.5em;background-color:#fff;text-align:center}.markdown-body.slides section[data-markdown] code{text-align:left}.markdown-body.slides section[data-markdown]:before{content:"";display:block;padding-bottom:56.23%}.markdown-body.slides section[data-markdown]>div:first-child{position:absolute;top:50%;left:1em;right:1em;transform:translateY(-50%);max-height:100%;overflow:hidden}.markdown-body.slides section[data-markdown]>ul{display:inline-block}.markdown-body.slides>section>section+section:after{content:"";position:absolute;top:-1.5em;right:1em;height:1.5em;border:3px solid #777}body{font-smoothing:subpixel-antialiased!important;-webkit-font-smoothing:subpixel-antialiased!important;-moz-osx-font-smoothing:auto!important;text-shadow:0 0 1em transparent,1px 1px 1.2px rgba(0,0,0,.004);-webkit-overflow-scrolling:touch;letter-spacing:.025em}.focus,:focus{outline:none!important}::-moz-focus-inner{border:0!important}body{font-family:Source Sans Pro,Helvetica,Arial,sans-serif}html[lang^=ja] body{font-family:Source Sans Pro,Helvetica,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html[lang=zh-tw] body{font-family:Source Sans Pro,Helvetica,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html[lang=zh-cn] body{font-family:Source Sans Pro,Helvetica,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}abbr[title]{border-bottom:none;text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted}abbr[data-original-title],abbr[title]{cursor:help}body.modal-open{overflow-y:auto;padding-right:0!important}
    </style>
    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
    	<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js" integrity="sha256-3Jy/GbSLrg0o9y5Z5n1uw0qxZECH7C6OQpVBgNFYa0g=" crossorigin="anonymous"></script>
    	<script src="https://cdnjs.cloudflare.com/ajax/libs/respond.js/1.4.2/respond.min.js" integrity="sha256-g6iAfvZp+nDQ2TdTR/VVKJf3bGro4ub5fvWSWVRi2NE=" crossorigin="anonymous"></script>
		<script src="https://cdnjs.cloudflare.com/ajax/libs/es5-shim/4.5.9/es5-shim.min.js" integrity="sha256-8E4Is26QH0bD52WoQpcB+R/tcWQtpzlCojrybUd7Mxo=" crossorigin="anonymous"></script>
    <![endif]-->
</head>

<body>
    <div id="doc" class="markdown-body container-fluid comment-enabled" data-hard-breaks="true" style="position: relative;"><h1 id="Implementation-of-AlexNet"><a class="anchor hidden-xs" href="#Implementation-of-AlexNet" title="Implementation-of-AlexNet"><span class="octicon octicon-link"></span></a>Implementation of AlexNet</h1><p><br><span style="font-size:14pt;">We will use the tensorflow.keras Functional API to build AlexNet</span></p><p>(<a href="https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf" target="_blank" rel="noopener">https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf</a>)</p><hr><p>In the paper we can read:</p><blockquote>
<p><strong>[i]</strong> “The ReLU non-linearity is applied to the output of every convolutional and fully-connected layer.”</p>
<p><strong>[ii]</strong> “We applied this normalization after applying the ReLU nonlinearity in certain layers.”</p>
<p><strong>[iii]</strong> “If we set s &lt; z, we obtain overlapping pooling. This is what we use throughout our network, with s = 2 and z = 3.”</p>
<p><strong>[iv]</strong> “The first convolutional layer filters the 224×224×3 input image with 96 kernels of size 11×11×3 with a stride of 4 pixels (this is the distance between the receptive field centers of neighboring neurons in a kernel map)."</p>
<p><strong>[v]</strong> "The second convolutional layer takes as input the (response-normalized and pooled) output of the first convolutional layer and filters it with 256 kernels of size 5 × 5 × 48.</p>
<p><strong>[vi]</strong> "The third, fourth, and fifth convolutional layers are connected to one another without any intervening pooling or normalization layers. The third convolutional layer has 384 kernels of size 3 × 3 × 256 connected to the (normalized, pooled) outputs of the second convolutional layer.”</p>
<p><strong>[vii]</strong> ”The fourth convolutional layer has 384 kernels of size 3 × 3 × 192 , and the fifth convolutional layer has 256 kernels of size 3 × 3 × 192. The fully-connected layers have 4096 neurons each.”</p>
<p><strong>[viii]</strong> “We use dropout in the first two fully-connected layers […]”</p>
</blockquote><br><p>We will also use the following Diagram <strong>[ix]</strong>:</p><img src="https://raw.githubusercontent.com/Machine-Learning-Tokyo/DL-workshop-series/master/Part%20I%20-%20Convolution%20Operations/images/AlexNet.png" width="800"><hr><h2 id="Network-architecture"><a class="anchor hidden-xs" href="#Network-architecture" title="Network-architecture"><span class="octicon octicon-link"></span></a>Network architecture</h2><ul>
<li>
<p>The network consists of 5 <em>Convolutional</em> layers and 3 <em>Fully Connected</em> Layers (<strong>[ix]</strong>)</p>
</li>
<li>
<p><em>Max Pooling</em> is applied Between the layers:</p>
</li>
<li>
<p>1conv-2conv (<strong>[v]</strong>)</p>
</li>
<li>
<p>2conv-3conv (<strong>[vi]</strong>)</p>
</li>
<li>
<p>5conv-1fc (<strong>[ix]</strong>)</p>
</li>
<li>
<p>Before Max Pooling a normalization technique is applied. At the paper a normalization method named LRN (Local Response Normalization) was used. However, since LRN is not part of the standard <em>tensorflow.keras</em> library and it is not in the scope of this section to teach how to write custom layers, we will use another method instead. We chose to replace LRN with <em>Batch Normalization</em> for this example.</p>
</li>
</ul><hr><h2 id="Workflow"><a class="anchor hidden-xs" href="#Workflow" title="Workflow"><span class="octicon octicon-link"></span></a>Workflow</h2><p>We will:</p><ol>
<li>import the neccesary layers</li>
<li>demonstrate how tensors and tf.keras layers work</li>
<li>write the code for the first block</li>
<li>write the code for the blocks 2-5 (<em>Convolution blocks</em>)</li>
<li>write the code for the hidden <em>Fully Connected blocks</em></li>
<li>write the code for the output <em>Fully Connected blocks</em></li>
<li>build the model</li>
</ol><hr><h3 id="1-Imports"><a class="anchor hidden-xs" href="#1-Imports" title="1-Imports"><span class="octicon octicon-link"></span></a>1. Imports</h3><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-keyword">from</span> tensorflow.keras.layers <span class="hljs-keyword">import</span> Input, Conv2D, \
     BatchNormalization, MaxPool2D, Flatten, Dense, Dropout
</code></pre>
</blockquote><hr><h3 id="2-Tensors-and-Layers"><a class="anchor hidden-xs" href="#2-Tensors-and-Layers" title="2-Tensors-and-Layers"><span class="octicon octicon-link"></span></a>2. <em>Tensors</em> and <em>Layers</em></h3><p>The idea is to build a graph of tensors which are connected through layers. We will start with the input tensor which is created by an <code>Input()</code> layer.</p><p>In the <code>Input()</code> layer we have to define the <em>shape</em> of the input object (e.g. a numpy array).</p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">input = Input(shape=(<span class="hljs-number">224</span>, <span class="hljs-number">224</span>, <span class="hljs-number">3</span>))
</code></pre>
</blockquote><p><code>input</code> is a tensorflow Tensor object:</p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-meta">&gt;&gt;&gt; </span>type(input)
tensorflow.python.framework.ops.Tensor
</code></pre>
</blockquote><hr><h3 id="3-1st-block"><a class="anchor hidden-xs" href="#3-1st-block" title="3-1st-block"><span class="octicon octicon-link"></span></a>3. 1st block</h3><p>Now we can pass this tensor to another <em>layer</em> object and the output will be another tensor.</p><p>The first layer that we will apply is a <em>Convolutional</em> layer (<code>Conv2D()</code>)</p><p>In this layer we will pass:</p><ul>
<li>the number of filters</li>
<li>the kernel size</li>
<li>the strides</li>
<li>the padding</li>
<li>the activation function</li>
</ul><p>From the paper:</p><blockquote>
<p>The first convolutional layer filters the 224×224×3 input image with <strong>96 kernels</strong> of size <strong>11×11</strong>×3 with a <strong>stride of 4</strong> pixels (<strong>[iv]</strong>)</p>
</blockquote><blockquote>
<p>The <strong>ReLU</strong> non-linearity is applied to the output of every convolutional and fully-connected layer (<strong>[i]</strong>)</p>
</blockquote><p><em>Note: at each reference we mark with <strong>bold</strong> the parts that are informative for the coding of the corresponding block.</em></p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = Conv2D(filters=<span class="hljs-number">96</span>,
          kernel_size=<span class="hljs-number">11</span>,
          strides=<span class="hljs-number">4</span>,
          padding=<span class="hljs-string">'same'</span>,
          activation=<span class="hljs-string">'relu'</span>)(input)  <span class="hljs-comment"># 1st convolutional layer</span>
</code></pre>
</blockquote><br><p>Before the second Convolutional layer we have to normalize and pool the output of the first Convolutional layer:</p><blockquote>
<p>“The second convolutional layer takes as input the (<strong>response-normalized and pooled</strong>) output of the first convolutional layer” (<strong>[v]</strong>)</p>
</blockquote><p>Regarding the <code>MaxPool2D()</code> layer we have to pass:</p><ul>
<li>the pool size</li>
<li>the strides</li>
</ul><p>From the paper:</p><blockquote>
<p>If we set s &lt; z, we obtain overlapping pooling. This is what we use throughout our network, with <strong>s = 2 and z = 3</strong> (<strong>[iii]</strong>)</p>
</blockquote><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = BatchNormalization()(x)
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)(x)
</code></pre>
</blockquote><br><p>Notice that it is ok to use the same name <code>x</code> for the output tensor of different layers as long as we implement with the correct order.</p><br><p>So the first block is now:</p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = Conv2D(filters=<span class="hljs-number">96</span>,
          kernel_size=<span class="hljs-number">11</span>,
          strides=<span class="hljs-number">4</span>,
          padding=<span class="hljs-string">'same'</span>,
          activation=<span class="hljs-string">'relu'</span>)(input)  <span class="hljs-comment"># 1st convolutional layer</span>

x = BatchNormalization()(x)
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)(x)
</code></pre>
</blockquote><hr><h3 id="4-2nd-5th-block"><a class="anchor hidden-xs" href="#4-2nd-5th-block" title="4-2nd-5th-block"><span class="octicon octicon-link"></span></a>4. 2nd-5th block</h3><p>From the paper:</p><blockquote>
<p>The <strong>second</strong> convolutional layer takes as input the (response-normalized and pooled) output of the first convolutional layer and filters it with <strong>256 kernels of size 5 × 5</strong> × 48 (<strong>[v]</strong>)</p>
</blockquote><p><em>Note: The <strong>48</strong> as the size of the last dimension of the kernel is technically 96 in our case. This is because the input tensor has 96 feature maps, equal to the number of kernels of the previous layer. However, since the original network was implemented in two GPUs in parallel, each one had 48 feature maps at this stage.</em></p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = Conv2D(filters=<span class="hljs-number">256</span>,
          kernel_size=<span class="hljs-number">5</span>,
          padding=<span class="hljs-string">'same'</span>,
          activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 2nd convolutional layer</span>
</code></pre>
</blockquote><br><p>From the paper:</p><blockquote>
<p>The <strong>third</strong> convolutional layer has <strong>384 kernels of size 3 × 3</strong> × 256 connected to the (<strong>normalized, pooled</strong>) outputs of the second convolutional layer. (<strong>[vi]</strong>)</p>
</blockquote><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = BatchNormalization()(x)
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)(x)

x = Conv2D(filters=<span class="hljs-number">384</span>,
          kernel_size=<span class="hljs-number">3</span>,
          padding=<span class="hljs-string">'same'</span>,
          activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 3rd convolutional layer</span>
</code></pre>
</blockquote><br><p>From the paper:</p><blockquote>
<p>The <strong>fourth</strong> convolutional layer has <strong>384 kernels of size 3 × 3</strong> × 192 (<strong>[vii]</strong>)</p>
</blockquote><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = Conv2D(filters=<span class="hljs-number">384</span>,
          kernel_size=<span class="hljs-number">3</span>,
          padding=<span class="hljs-string">'same'</span>,
          activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 4th convolutional layer</span>
</code></pre>
</blockquote><br><p>From the paper:</p><blockquote>
<p>the <strong>fifth</strong> convolutional layer has <strong>256 kernels of size 3 × 3</strong> × 192. (<strong>[vii]</strong>)</p>
</blockquote><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = Conv2D(filters=<span class="hljs-number">256</span>,
          kernel_size=<span class="hljs-number">3</span>,
          padding=<span class="hljs-string">'same'</span>,
          activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 5th convolutional layer</span>

x = BatchNormalization()(x)
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)(x)
</code></pre>
</blockquote><hr><h3 id="5-Dense-layers"><a class="anchor hidden-xs" href="#5-Dense-layers" title="5-Dense-layers"><span class="octicon octicon-link"></span></a>5. Dense layers</h3><p>Before passing the output tensor of the last Convolutional layer (13x13x256) to the first <code>Dense()</code> layer we have to flatten it to a one-dimension tensor. We do it by using the <code>Flatten()</code> layer.</p><p><em>Note: you may find the (one-dimension) tensors to also be called vectors. We use the word tensor as reference to the tensorflow.Tensor object of which it is an instance.</em></p><p>From the paper:</p><blockquote>
<p>The <strong>fully-connected</strong> layers have <strong>4096</strong> neurons each (<strong>[vii]</strong>)<br>
Dropout with drop probability 0.5 is used after the first 2 Fully Connected layers (<strong>[viii]</strong>)</p>
</blockquote><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">x = Flatten()(x)
x = Dense(units=<span class="hljs-number">4096</span>, activation=<span class="hljs-string">'relu'</span>)(x)
x = Dense(units=<span class="hljs-number">4096</span>, activation=<span class="hljs-string">'relu'</span>)(x)
x = Dropout(rate=<span class="hljs-number">0.5</span>)(x)
</code></pre>
</blockquote><hr><h3 id="6-Output-layer"><a class="anchor hidden-xs" href="#6-Output-layer" title="6-Output-layer"><span class="octicon octicon-link"></span></a>6. Output layer</h3><p>Since the model is to be used for classifiction tasks, the output of the model will be a <code>Dense()</code> layer with:</p><ul>
<li>number of units equal to the number of classes in our task which are 1000 based on <strong>[ix]</strong></li>
<li><code>softmax</code> actication if we target to only one class per image</li>
</ul><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">output = Dense(units=<span class="hljs-number">1000</span>, activation=<span class="hljs-string">'softmax'</span>)(x)
</code></pre>
</blockquote><hr><h3 id="7-Model"><a class="anchor hidden-xs" href="#7-Model" title="7-Model"><span class="octicon octicon-link"></span></a>7. Model</h3><p>In order to build the <em>model</em> we will use the <code>tensorflow.keras.Model</code> object:</p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs"><span class="hljs-keyword">from</span> tensorflow.keras <span class="hljs-keyword">import</span> Model
</code></pre>
</blockquote><p>To define the model we need the input tensor(s) and the output tensor(s).</p><p><strong>Code:</strong></p><blockquote>
<pre><code class="python hljs">model = Model(inputs=input, outputs=output)
</code></pre>
</blockquote><hr><h2 id="Final-code"><a class="anchor hidden-xs" href="#Final-code" title="Final-code"><span class="octicon octicon-link"></span></a>Final code</h2><p><strong>Code:</strong></p><pre><code class="python hljs"><span class="hljs-keyword">from</span> tensorflow.keras.layers <span class="hljs-keyword">import</span> Input, Conv2D, \
     BatchNormalization, MaxPool2D, Flatten, Dense, Dropout

input = Input(shape=(<span class="hljs-number">224</span>, <span class="hljs-number">224</span>, <span class="hljs-number">3</span>))

x = Conv2D(filters=<span class="hljs-number">96</span>,
           kernel_size=<span class="hljs-number">11</span>,
           strides=<span class="hljs-number">4</span>,
           padding=<span class="hljs-string">'same'</span>,
           activation=<span class="hljs-string">'relu'</span>)(input)  <span class="hljs-comment"># 1st convolutional layer</span>
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)(x)

x = Conv2D(filters=<span class="hljs-number">256</span>,
           kernel_size=<span class="hljs-number">5</span>,
           padding=<span class="hljs-string">'same'</span>,
           activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 2nd convolutional layer</span>
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)(x)

x = Conv2D(filters=<span class="hljs-number">384</span>,
           kernel_size=<span class="hljs-number">3</span>,
           padding=<span class="hljs-string">'same'</span>,
           activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 3rd convolutional layer</span>

x = Conv2D(filters=<span class="hljs-number">384</span>,
           kernel_size=<span class="hljs-number">3</span>,
           padding=<span class="hljs-string">'same'</span>,
           activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 4th convolutional layer</span>

x = Conv2D(filters=<span class="hljs-number">256</span>,
           kernel_size=<span class="hljs-number">3</span>,
           padding=<span class="hljs-string">'same'</span>,
           activation=<span class="hljs-string">'relu'</span>)(x)  <span class="hljs-comment"># 5th convolutional layer</span>
x = BatchNormalization()(x)
x = MaxPool2D(pool_size=<span class="hljs-number">3</span>, strides=<span class="hljs-number">2</span>)(x)

x = Flatten()(x)
x = Dense(units=<span class="hljs-number">4096</span>, activation=<span class="hljs-string">'relu'</span>)(x)
x = Dense(units=<span class="hljs-number">4096</span>, activation=<span class="hljs-string">'relu'</span>)(x)
x = Dropout(rate=<span class="hljs-number">0.5</span>)(x)

output = Dense(units=<span class="hljs-number">1000</span>, activation=<span class="hljs-string">'softmax'</span>)(x)

<span class="hljs-keyword">from</span> tensorflow.keras <span class="hljs-keyword">import</span> Model

model = Model(inputs=input, outputs=output)
</code></pre><hr><h2 id="Model-diagram"><a class="anchor hidden-xs" href="#Model-diagram" title="Model-diagram"><span class="octicon octicon-link"></span></a>Model diagram</h2><img src="https://raw.githubusercontent.com/Machine-Learning-Tokyo/CNN-Architectures/master/Implementations/AlexNet/AlexNet_diagram.svg?sanitize=true"></div>
    <div class="ui-toc dropup unselectable hidden-print" style="display:none;">
        <div class="pull-right dropdown">
            <a id="tocLabel" class="ui-toc-label btn btn-default" data-toggle="dropdown" href="#" role="button" aria-haspopup="true" aria-expanded="false" title="Table of content">
                <i class="fa fa-bars"></i>
            </a>
            <ul id="ui-toc" class="ui-toc-dropdown dropdown-menu" aria-labelledby="tocLabel">
                <div class="toc"><ul class="nav">
<li class=""><a href="#Implementation-of-AlexNet" title="Implementation of AlexNet">Implementation of AlexNet</a><ul class="nav">
<li><a href="#Network-architecture" title="Network architecture">Network architecture</a></li>
<li><a href="#Workflow" title="Workflow">Workflow</a><ul class="nav">
<li><a href="#1-Imports" title="1. Imports">1. Imports</a></li>
<li><a href="#2-Tensors-and-Layers" title="2. Tensors and Layers">2. Tensors and Layers</a></li>
<li><a href="#3-1st-block" title="3. 1st block">3. 1st block</a></li>
<li><a href="#4-2nd-5th-block" title="4. 2nd-5th block">4. 2nd-5th block</a></li>
<li><a href="#5-Dense-layers" title="5. Dense layers">5. Dense layers</a></li>
<li><a href="#6-Output-layer" title="6. Output layer">6. Output layer</a></li>
<li><a href="#7-Model" title="7. Model">7. Model</a></li>
</ul>
</li>
<li class=""><a href="#Final-code" title="Final code">Final code</a></li>
<li class=""><a href="#Model-diagram" title="Model diagram">Model diagram</a></li>
</ul>
</li>
</ul>
</div><div class="toc-menu"><a class="expand-toggle" href="#">Expand all</a><a class="back-to-top" href="#">Back to top</a><a class="go-to-bottom" href="#">Go to bottom</a></div>
            </ul>
        </div>
    </div>
    <div id="ui-toc-affix" class="ui-affix-toc ui-toc-dropdown unselectable hidden-print" data-spy="affix" style="top:17px;display:none;" null null>
        <div class="toc"><ul class="nav">
<li class=""><a href="#Implementation-of-AlexNet" title="Implementation of AlexNet">Implementation of AlexNet</a><ul class="nav">
<li><a href="#Network-architecture" title="Network architecture">Network architecture</a></li>
<li><a href="#Workflow" title="Workflow">Workflow</a><ul class="nav">
<li><a href="#1-Imports" title="1. Imports">1. Imports</a></li>
<li><a href="#2-Tensors-and-Layers" title="2. Tensors and Layers">2. Tensors and Layers</a></li>
<li><a href="#3-1st-block" title="3. 1st block">3. 1st block</a></li>
<li><a href="#4-2nd-5th-block" title="4. 2nd-5th block">4. 2nd-5th block</a></li>
<li><a href="#5-Dense-layers" title="5. Dense layers">5. Dense layers</a></li>
<li><a href="#6-Output-layer" title="6. Output layer">6. Output layer</a></li>
<li><a href="#7-Model" title="7. Model">7. Model</a></li>
</ul>
</li>
<li class=""><a href="#Final-code" title="Final code">Final code</a></li>
<li class=""><a href="#Model-diagram" title="Model diagram">Model diagram</a></li>
</ul>
</li>
</ul>
</div><div class="toc-menu"><a class="expand-toggle" href="#">Expand all</a><a class="back-to-top" href="#">Back to top</a><a class="go-to-bottom" href="#">Go to bottom</a></div>
    </div>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js" integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous" defer></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/gist-embed/2.6.0/gist-embed.min.js" integrity="sha256-KyF2D6xPIJUW5sUDSs93vWyZm+1RzIpKCexxElmxl8g=" crossorigin="anonymous" defer></script>
    <script>
        var markdown = $(".markdown-body");
        //smooth all hash trigger scrolling
        function smoothHashScroll() {
            var hashElements = $("a[href^='#']").toArray();
            for (var i = 0; i < hashElements.length; i++) {
                var element = hashElements[i];
                var $element = $(element);
                var hash = element.hash;
                if (hash) {
                    $element.on('click', function (e) {
                        // store hash
                        var hash = this.hash;
                        if ($(hash).length <= 0) return;
                        // prevent default anchor click behavior
                        e.preventDefault();
                        // animate
                        $('body, html').stop(true, true).animate({
                            scrollTop: $(hash).offset().top
                        }, 100, "linear", function () {
                            // when done, add hash to url
                            // (default click behaviour)
                            window.location.hash = hash;
                        });
                    });
                }
            }
        }

        smoothHashScroll();
        var toc = $('.ui-toc');
        var tocAffix = $('.ui-affix-toc');
        var tocDropdown = $('.ui-toc-dropdown');
        //toc
        tocDropdown.click(function (e) {
            e.stopPropagation();
        });

        var enoughForAffixToc = true;

        function generateScrollspy() {
            $(document.body).scrollspy({
                target: ''
            });
            $(document.body).scrollspy('refresh');
            if (enoughForAffixToc) {
                toc.hide();
                tocAffix.show();
            } else {
                tocAffix.hide();
                toc.show();
            }
            $(document.body).scroll();
        }

        function windowResize() {
            //toc right
            var paddingRight = parseFloat(markdown.css('padding-right'));
            var right = ($(window).width() - (markdown.offset().left + markdown.outerWidth() - paddingRight));
            toc.css('right', right + 'px');
            //affix toc left
            var newbool;
            var rightMargin = (markdown.parent().outerWidth() - markdown.outerWidth()) / 2;
            //for ipad or wider device
            if (rightMargin >= 133) {
                newbool = true;
                var affixLeftMargin = (tocAffix.outerWidth() - tocAffix.width()) / 2;
                var left = markdown.offset().left + markdown.outerWidth() - affixLeftMargin;
                tocAffix.css('left', left + 'px');
            } else {
                newbool = false;
            }
            if (newbool != enoughForAffixToc) {
                enoughForAffixToc = newbool;
                generateScrollspy();
            }
        }
        $(window).resize(function () {
            windowResize();
        });
        $(document).ready(function () {
            windowResize();
            generateScrollspy();
        });

        //remove hash
        function removeHash() {
            window.location.hash = '';
        }

        var backtotop = $('.back-to-top');
        var gotobottom = $('.go-to-bottom');

        backtotop.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            if (scrollToTop)
                scrollToTop();
            removeHash();
        });
        gotobottom.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            if (scrollToBottom)
                scrollToBottom();
            removeHash();
        });

        var toggle = $('.expand-toggle');
        var tocExpand = false;

        checkExpandToggle();
        toggle.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            tocExpand = !tocExpand;
            checkExpandToggle();
        })

        function checkExpandToggle () {
            var toc = $('.ui-toc-dropdown .toc');
            var toggle = $('.expand-toggle');
            if (!tocExpand) {
                toc.removeClass('expand');
                toggle.text('Expand all');
            } else {
                toc.addClass('expand');
                toggle.text('Collapse all');
            }
        }

        function scrollToTop() {
            $('body, html').stop(true, true).animate({
                scrollTop: 0
            }, 100, "linear");
        }

        function scrollToBottom() {
            $('body, html').stop(true, true).animate({
                scrollTop: $(document.body)[0].scrollHeight
            }, 100, "linear");
        }
    </script>
</body>

</html>
